--- title: Recreating Steffl's Calib keywords: fastai sidebar: home_sidebar summary: "Based on his thesis appendix" description: "Based on his thesis appendix" nb_path: "notebooks/06_calib.steffl.ipynb" ---
{% raw %}
{% endraw %} {% raw %}
{% endraw %} {% raw %} 
['2001-093', '2002-198', '2003-139']
{% endraw %} {% raw %} 
steffl_spica_dates[2]

'2003-05-19'
{% endraw %} {% raw %} 
filter_spica_for_date(steffl_spica_dates[2])
filename slit nx ny nz int odcid x1 x2 y1 y2 name detector filename_time date
246 EUV2003_139_18_12_02_UVIS_C37ST_SPICARAST001_P... 2 1024 64 10 332 120 0 1023 0 63 alp vir EUV 2003-05-19 18:12:02 2003-05-19
247 EUV2003_139_19_08_14_UVIS_C37ST_SPICARAST001_P... 2 1024 64 10 332 120 0 1023 0 63 alp vir EUV 2003-05-19 19:08:14 2003-05-19
248 EUV2003_139_20_04_26_UVIS_C37ST_SPICARAST001_P... 2 1024 64 10 332 120 0 1023 0 63 alp vir EUV 2003-05-19 20:04:26 2003-05-19
249 EUV2003_139_21_00_38_UVIS_C37ST_SPICARAST001_P... 2 1024 64 10 332 120 0 1023 0 63 alp vir EUV 2003-05-19 21:00:38 2003-05-19
250 EUV2003_139_21_56_50_UVIS_C37ST_SPICARAST001_P... 2 1024 64 10 332 120 0 1023 0 63 alp vir EUV 2003-05-19 21:56:50 2003-05-19
251 EUV2003_139_22_53_02_UVIS_C37ST_SPICARAST001_P... 2 1024 64 10 332 120 0 1023 0 63 alp vir EUV 2003-05-19 22:53:02 2003-05-19
252 EUV2003_139_23_49_14_UVIS_C37ST_SPICARAST001_P... 2 1024 64 10 332 120 0 1023 0 63 alp vir EUV 2003-05-19 23:49:14 2003-05-19
930 FUV2003_139_18_12_02_UVIS_C37ST_SPICARAST001_P... 1 1024 64 10 332 120 0 1023 0 63 alp vir FUV 2003-05-19 18:12:02 2003-05-19
931 FUV2003_139_19_08_14_UVIS_C37ST_SPICARAST001_P... 1 1024 64 10 332 120 0 1023 0 63 alp vir FUV 2003-05-19 19:08:14 2003-05-19
932 FUV2003_139_20_04_26_UVIS_C37ST_SPICARAST001_P... 1 1024 64 10 332 120 0 1023 0 63 alp vir FUV 2003-05-19 20:04:26 2003-05-19
933 FUV2003_139_21_00_38_UVIS_C37ST_SPICARAST001_P... 1 1024 64 10 332 120 0 1023 0 63 alp vir FUV 2003-05-19 21:00:38 2003-05-19
934 FUV2003_139_21_56_50_UVIS_C37ST_SPICARAST001_P... 1 1024 64 10 332 120 0 1023 0 63 alp vir FUV 2003-05-19 21:56:50 2003-05-19
935 FUV2003_139_22_53_02_UVIS_C37ST_SPICARAST001_P... 1 1024 64 10 332 120 0 1023 0 63 alp vir FUV 2003-05-19 22:53:02 2003-05-19
936 FUV2003_139_23_49_14_UVIS_C37ST_SPICARAST001_P... 1 1024 64 10 332 120 0 1023 0 63 alp vir FUV 2003-05-19 23:49:14 2003-05-19
{% endraw %} {% raw %} 
obsids = filter_spica_for_date(steffl_spica_dates[1]).filename.values
obsids

array(['EUV2002_198_03_26_54_UVIS_C33ST_SPICARAST002_PRIME',
       'EUV2002_198_04_32_26_UVIS_C33ST_SPICARAST002_PRIME',
       'EUV2002_198_05_28_38_UVIS_C33ST_SPICARAST002_PRIME',
       'EUV2002_198_06_24_49_UVIS_C33ST_SPICARAST002_PRIME',
       'EUV2002_198_07_21_01_UVIS_C33ST_SPICARAST002_PRIME',
       'EUV2002_198_08_17_13_UVIS_C33ST_SPICARAST002_PRIME',
       'EUV2002_198_09_13_25_UVIS_C33ST_SPICARAST002_PRIME',
       'EUV2002_198_10_09_38_UVIS_C33ST_SPICARAST002_PRIME',
       'EUV2002_198_11_05_50_UVIS_C33ST_SPICARAST002_PRIME',
       'EUV2002_198_12_02_01_UVIS_C33ST_SPICARAST002_PRIME',
       'EUV2002_198_12_58_13_UVIS_C33ST_SPICARAST002_PRIME',
       'EUV2002_198_13_14_49_UVIS_C33ST_SPICARAST002_PRIME',
       'EUV2002_198_13_54_25_UVIS_C33ST_SPICARAST002_PRIME',
       'FUV2002_198_03_26_54_UVIS_C33ST_SPICARAST002_PRIME',
       'FUV2002_198_04_32_26_UVIS_C33ST_SPICARAST002_PRIME',
       'FUV2002_198_05_28_38_UVIS_C33ST_SPICARAST002_PRIME',
       'FUV2002_198_06_24_49_UVIS_C33ST_SPICARAST002_PRIME',
       'FUV2002_198_07_21_01_UVIS_C33ST_SPICARAST002_PRIME',
       'FUV2002_198_08_17_13_UVIS_C33ST_SPICARAST002_PRIME',
       'FUV2002_198_09_13_25_UVIS_C33ST_SPICARAST002_PRIME',
       'FUV2002_198_10_09_38_UVIS_C33ST_SPICARAST002_PRIME',
       'FUV2002_198_11_05_50_UVIS_C33ST_SPICARAST002_PRIME',
       'FUV2002_198_12_02_01_UVIS_C33ST_SPICARAST002_PRIME',
       'FUV2002_198_12_58_13_UVIS_C33ST_SPICARAST002_PRIME',
       'FUV2002_198_13_14_49_UVIS_C33ST_SPICARAST002_PRIME',
       'FUV2002_198_13_54_25_UVIS_C33ST_SPICARAST002_PRIME'], dtype=object)
{% endraw %} {% raw %} 
for id in obsids:
    try:
        data = UVPDS(id)
    except FileNotFoundError:
        print(id, "not there.")
    else:
        print("Got", id)

Got EUV2002_198_03_26_54_UVIS_C33ST_SPICARAST002_PRIME
EUV2002_198_04_32_26_UVIS_C33ST_SPICARAST002_PRIME not there.
Got EUV2002_198_05_28_38_UVIS_C33ST_SPICARAST002_PRIME
Got EUV2002_198_06_24_49_UVIS_C33ST_SPICARAST002_PRIME
EUV2002_198_07_21_01_UVIS_C33ST_SPICARAST002_PRIME not there.
EUV2002_198_08_17_13_UVIS_C33ST_SPICARAST002_PRIME not there.
EUV2002_198_09_13_25_UVIS_C33ST_SPICARAST002_PRIME not there.
Got EUV2002_198_10_09_38_UVIS_C33ST_SPICARAST002_PRIME
Got EUV2002_198_11_05_50_UVIS_C33ST_SPICARAST002_PRIME
EUV2002_198_12_02_01_UVIS_C33ST_SPICARAST002_PRIME not there.
EUV2002_198_12_58_13_UVIS_C33ST_SPICARAST002_PRIME not there.
Got EUV2002_198_13_14_49_UVIS_C33ST_SPICARAST002_PRIME
EUV2002_198_13_54_25_UVIS_C33ST_SPICARAST002_PRIME not there.
Got FUV2002_198_03_26_54_UVIS_C33ST_SPICARAST002_PRIME
FUV2002_198_04_32_26_UVIS_C33ST_SPICARAST002_PRIME not there.
Got FUV2002_198_05_28_38_UVIS_C33ST_SPICARAST002_PRIME
Got FUV2002_198_06_24_49_UVIS_C33ST_SPICARAST002_PRIME
FUV2002_198_07_21_01_UVIS_C33ST_SPICARAST002_PRIME not there.
FUV2002_198_08_17_13_UVIS_C33ST_SPICARAST002_PRIME not there.
FUV2002_198_09_13_25_UVIS_C33ST_SPICARAST002_PRIME not there.
Got FUV2002_198_10_09_38_UVIS_C33ST_SPICARAST002_PRIME
Got FUV2002_198_11_05_50_UVIS_C33ST_SPICARAST002_PRIME
FUV2002_198_12_02_01_UVIS_C33ST_SPICARAST002_PRIME not there.
FUV2002_198_12_58_13_UVIS_C33ST_SPICARAST002_PRIME not there.
Got FUV2002_198_13_14_49_UVIS_C33ST_SPICARAST002_PRIME
FUV2002_198_13_54_25_UVIS_C33ST_SPICARAST002_PRIME not there.
{% endraw %} {% raw %} 
import hvplot.xarray

kwargs = {"x": "nx", "y": "ny", "cmap": "viridis", "clim": (0, 50)}
{% endraw %} {% raw %} 
d1 = EFUV_PDS("EUV2002_198_04_31")
d2 = EFUV_PDS(obsids[3])
{% endraw %} {% raw %} 
d1.label

PVLModule([
  ('PDS_VERSION_ID', 'PDS3')
  ('RECORD_TYPE', 'FIXED_LENGTH')
  ('RECORD_BYTES', 131072)
  ('FILE_RECORDS', 10)
  ('PRODUCT_ID', 'EUV2002_198_04_31')
  ('SPACECRAFT_CLOCK_START_COUNT', '1/1405572559.96')
  ('SPACECRAFT_CLOCK_STOP_COUNT', 'UNK')
  ('START_TIME',
   datetime.datetime(2002, 7, 17, 4, 31, 59, 571000, tzinfo=datetime.timezone.utc))
  ('STOP_TIME',
   datetime.datetime(2002, 7, 17, 5, 27, 19, 571000, tzinfo=datetime.timezone.utc))
  ('DATA_SET_ID', 'CO-S-UVIS-2-CUBE-V1.2')
  ('TARGET_NAME', 'STAR')
  ('INSTRUMENT_HOST_NAME', 'CASSINI_ORBITER')
  ('INSTRUMENT_NAME', 'ULTRAVIOLET IMAGING SPECTROGRAPH')
  ('INSTRUMENT_ID', 'UVIS')
  ('OBSERVATION_ID', 24440)
  ('MISSION_PHASE_NAME', 'CRUISE')
  ('PRODUCT_CREATION_TIME', datetime.date(2012, 1, 25))
  ('INTEGRATION_DURATION', Quantity(value=332.0, units='SECOND'))
  ('COMPRESSION_TYPE', 'SQRT_9')
  ('HI_VOLTAGE_POWER_SUPPLY_STATE', 'ON')
  ('OCCULTATION_PORT_STATE', 'CLOSED')
  ('SLIT_STATE', 'OCCULTATION')
  ('TEST_PULSE_STATE', 'OFF')
  ('ODC_ID', 120)
  ('DESCRIPTION',
   'This file was produced by the Cassini UVIS team at the Laboratory for '
   'Atmospheric and Space Physics (LASP) at the University of Colorado at '
   'Boulder. For a description of hardware and scientific objectives, refer to '
   'the Cassini UVIS instrument paper The Cassini Ultraviolet Imaging '
   'Spectrograph Investigation, Space Science Reviews, 115, 299-361, by L. W. '
   'Esposito et. al.')
  ('RIGHT_ASCENSION', 'UNK')
  ('DECLINATION', 'UNK')
  ('SUB_SOLAR_LATITUDE', 'UNK')
  ('SUB_SOLAR_LONGITUDE', 'UNK')
  ('SUB_SPACECRAFT_LATITUDE', 'UNK')
  ('SUB_SPACECRAFT_LONGITUDE', 'UNK')
  ('PHASE_ANGLE', 'UNK')
  ('EMISSION_ANGLE', 'UNK')
  ('INCIDENCE_ANGLE', 'UNK')
  ('CENTRAL_BODY_DISTANCE', 'UNK')
  ('SC_PLANET_POSITION_VECTOR', 'UNK')
  ('SC_PLANET_VELOCITY_VECTOR', 'UNK')
  ('SC_SUN_POSITION_VECTOR',
   [Quantity(value=-13379440.0, units='KM'),
    Quantity(value=1033895900.0, units='KM'),
    Quantity(value=437807520.0, units='KM')])
  ('SC_SUN_VELOCITY_VECTOR',
   [Quantity(value=-6.3706824, units='KM/S'),
    Quantity(value=5.0657174, units='KM/S'),
    Quantity(value=2.2235017, units='KM/S')])
  ('SC_TARGET_POSITION_VECTOR', 'UNK')
  ('SC_TARGET_VELOCITY_VECTOR', 'UNK')
  ('PLANET_CENTER_POSITION_VECTOR', 'UNK')
  ('PLANET_CENTER_VELOCITY_VECTOR', 'UNK')
  ('^QUBE', 'EUV2002_198_04_31.DAT')
  ('QUBE',
   {'AXES': 3,
    'AXIS_NAME': ['BAND', 'LINE', 'SAMPLE'],
    'BAND_BIN': 1,
    'BAND_BIN_CENTER': 'N/A',
    'CORE_BASE': 0.0,
    'CORE_HIGH_INSTR_SATURATION': 'N/A',
    'CORE_HIGH_REPR_SATURATION': 'N/A',
    'CORE_ITEMS': [1024, 64, 10],
    'CORE_ITEM_BYTES': 2,
    'CORE_ITEM_TYPE': 'MSB_UNSIGNED_INTEGER',
    'CORE_LOW_INSTR_SATURATION': 'N/A',
    'CORE_LOW_REPR_SATURATION': 'N/A',
    'CORE_MULTIPLIER': 1.0,
    'CORE_NAME': 'RAW_DATA_NUMBER',
    'CORE_NULL': -1,
    'CORE_UNIT': 'COUNTS/BIN',
    'CORE_VALID_MINIMUM': 'N/A',
    'DESCRIPTION': 'This UVIS Qube is a 3 dimensional matrix of 2 byte integers. '
                   'The Qube represents a time series of two dimensional '
                   '(spectral x spatial) matrices. Each two dimensional matrix '
                   'is made up of counts taken by the detector during a fixed '
                   'time interval. The fixed time interval is specified in the '
                   'INTEGRATION_DURATION keyword. If the two dimensional matrix '
                   'has a LINE magnitude of 1 then the Qube is a time series of '
                   'spectra. The region of the detector in which data was taken '
                   'is defined by the ...CORNER keywords. This object is '
                   'organized so that a sequence of 1024 2-byte numbers forms a '
                   'horizontal line (the BAND dimension), a sequence of 64 lines '
                   'forms an BANDxLINE spectral-spatial matrix, and sequence of '
                   '10 of these matrices, forms a (1024x64x10) Qube. The purpose '
                   'of this observation is to measure the response of the EUV '
                   'and FUV channels to various stars for purposes of '
                   'photometric calibration.',
    'LINE_BIN': 1,
    'LR_CORNER_BAND': 1023,
    'LR_CORNER_LINE': 63,
    'SUFFIX_BYTES': 4,
    'SUFFIX_ITEMS': [0, 0, 0],
    'UL_CORNER_BAND': 0,
    'UL_CORNER_LINE': 0})
])
{% endraw %} {% raw %} 
d1arr = d1.xarray.astype("int16")
{% endraw %} {% raw %} 
d2arr = d2.xarray.astype("int16")
{% endraw %} {% raw %} 
diff = d2arr - d1arr
{% endraw %}

Background

{% raw %} 
background = d1arr.sel(ny=slice(50, 60), nx=slice(60, 70), nz=1, drop=True)
background

<xarray.DataArray 'EUV' (nx: 165, ny: 11)>
2 3 1 5 2 1 1 7 2 1 3 3 1 4 3 2 0 0 2 ... 2 2 3 3 3 3 2 0 2 5 4 3 2 3 1 3 2 3 2
Coordinates:
  * nx       (nx) float64 60.06 60.12 60.18 60.24 ... 69.81 69.87 69.93 69.99
  * ny       (ny) int64 50 51 52 53 54 55 56 57 58 59 60
Attributes:
    n_bands:               1024
    integration_duration:  Quantity(value=332.0, units='SECOND')
{% endraw %} {% raw %} 
background.hvplot(x="nx", y="ny", aspect=2)

WARNING:param.Image01562: Image dimension nx is  not evenly sampled to relative tolerance of 0.001. Please use the QuadMesh element for irregularly sampled data or set a higher tolerance on hv.config.image_rtol or the rtol parameter in the Image constructor.
WARNING:param.Image01562: Image dimension nx is  not evenly sampled to relative tolerance of 0.001. Please use the QuadMesh element for irregularly sampled data or set a higher tolerance on hv.config.image_rtol or the rtol parameter in the Image constructor.
{% endraw %} {% raw %} 
bps = background / 332
{% endraw %} {% raw %} 
bps.hvplot.hist()
{% endraw %} {% raw %} 
bps.mean()

<xarray.DataArray 'EUV' ()>
0.007636
Attributes:
    n_bands:               1024
    integration_duration:  Quantity(value=332.0, units='SECOND')
{% endraw %}

Column-averaging

{% raw %} 
class FlatFielder:
    def __init__(self, pid):
        self.pid = pid
        self.data = EFUV_PDS(pid).xarray.astype("int16")

    @property
    def integrated(self):
        return self.data.sum(dim="nz")

    @property
    def plot_integrated(self):
        return self.integrated.hvplot(x="nx", y="ny", cmap="viridis")

    @property
    def averaged(self):
        return self.integrated.sel(ny=slice(2, 60)).mean(dim="ny")

    @property
    def plot_averaged(self):
        return self.averaged.hvplot(x="nx")

    @property
    def ff(self):
        return self.integrated / self.averaged

    @property
    def plot_ff(self):
        return self.ff.hvplot(x="nx", y="ny", cmap="viridis")
{% endraw %} {% raw %} 
pid = "EUV2002_198_04_31"
pid = "EUV2003_139_21_00_38"
pid = "FUV2002_198_04_31"
pid = "FUV2003_139_21_00_38"
{% endraw %} {% raw %} 
flatter = FlatFielder(pid)
{% endraw %} {% raw %} 
flatter.plot_integrated

WARNING:param.Image07218: Image dimension nx is  not evenly sampled to relative tolerance of 0.001. Please use the QuadMesh element for irregularly sampled data or set a higher tolerance on hv.config.image_rtol or the rtol parameter in the Image constructor.
WARNING:param.Image07218: Image dimension nx is  not evenly sampled to relative tolerance of 0.001. Please use the QuadMesh element for irregularly sampled data or set a higher tolerance on hv.config.image_rtol or the rtol parameter in the Image constructor.
{% endraw %} {% raw %} 
flatter.plot_averaged
{% endraw %} {% raw %} 
flatter.data.hvplot(x="nx", y="ny", cmap="viridis")

WARNING:param.Image07626: Image dimension nx is  not evenly sampled to relative tolerance of 0.001. Please use the QuadMesh element for irregularly sampled data or set a higher tolerance on hv.config.image_rtol or the rtol parameter in the Image constructor.
WARNING:param.Image07626: Image dimension nx is  not evenly sampled to relative tolerance of 0.001. Please use the QuadMesh element for irregularly sampled data or set a higher tolerance on hv.config.image_rtol or the rtol parameter in the Image constructor.
{% endraw %} {% raw %} 
flatter.plot_ff

WARNING:param.Image07851: Image dimension nx is  not evenly sampled to relative tolerance of 0.001. Please use the QuadMesh element for irregularly sampled data or set a higher tolerance on hv.config.image_rtol or the rtol parameter in the Image constructor.
WARNING:param.Image07851: Image dimension nx is  not evenly sampled to relative tolerance of 0.001. Please use the QuadMesh element for irregularly sampled data or set a higher tolerance on hv.config.image_rtol or the rtol parameter in the Image constructor.
{% endraw %} {% raw %} 
diff.hvplot(x="nx", y="ny", cmap="viridis", clim=(-100, 100))

WARNING:param.Image38641: Image dimension nx is  not evenly sampled to relative tolerance of 0.001. Please use the QuadMesh element for irregularly sampled data or set a higher tolerance on hv.config.image_rtol or the rtol parameter in the Image constructor.
WARNING:param.Image38641: Image dimension nx is  not evenly sampled to relative tolerance of 0.001. Please use the QuadMesh element for irregularly sampled data or set a higher tolerance on hv.config.image_rtol or the rtol parameter in the Image constructor.
{% endraw %} {% raw %} 
d1.xarray.hvplot(x="nx", y="ny", cmap="viridis", clim=(0, 50))

WARNING:param.Image39466: Image dimension nx is  not evenly sampled to relative tolerance of 0.001. Please use the QuadMesh element for irregularly sampled data or set a higher tolerance on hv.config.image_rtol or the rtol parameter in the Image constructor.
WARNING:param.Image39466: Image dimension nx is  not evenly sampled to relative tolerance of 0.001. Please use the QuadMesh element for irregularly sampled data or set a higher tolerance on hv.config.image_rtol or the rtol parameter in the Image constructor.
{% endraw %} {% raw %} 
d2.xarray.hvplot(x="nx", y="ny", cmap="viridis", clim=(0, 50))

WARNING:param.Image39191: Image dimension nx is  not evenly sampled to relative tolerance of 0.001. Please use the QuadMesh element for irregularly sampled data or set a higher tolerance on hv.config.image_rtol or the rtol parameter in the Image constructor.
WARNING:param.Image39191: Image dimension nx is  not evenly sampled to relative tolerance of 0.001. Please use the QuadMesh element for irregularly sampled data or set a higher tolerance on hv.config.image_rtol or the rtol parameter in the Image constructor.
{% endraw %} {% raw %} 
(
    arr.hvplot(x="nx", y="ny", cmap="viridis", clim=(0, 50))
    + arr.hvplot.quadmesh(x="nx", y="ny", cmap="viridis", clim=(0, 50))
).cols(1)

WARNING:param.Image06919: Image dimension nx is  not evenly sampled to relative tolerance of 0.001. Please use the QuadMesh element for irregularly sampled data or set a higher tolerance on hv.config.image_rtol or the rtol parameter in the Image constructor.
WARNING:param.Image06919: Image dimension nx is  not evenly sampled to relative tolerance of 0.001. Please use the QuadMesh element for irregularly sampled data or set a higher tolerance on hv.config.image_rtol or the rtol parameter in the Image constructor.
{% endraw %} {% raw %} 
euv.integration_duration

Quantity(value=332.0, units='SECOND')
{% endraw %} {% raw %} 
diff = arr.nx.diff("nx")
diff.name = "diff"
{% endraw %} {% raw %} 
diff.hvplot.hist()
{% endraw %} {% raw %} 
fuv = FUV_PDS(UVISOpus(obsids[3][:-3]).local_label_path)

Found 1 obsids.
{% endraw %} {% raw %} 
fuvarr = fuv.xarray
{% endraw %} {% raw %} 
fuvarr.hvplot(x="nx", y="ny", clim=(0, 50), cmap="viridis")
{% endraw %} {% raw %} 
euv.label

PVLModule([
  ('PDS_VERSION_ID', 'PDS3')
  ('RECORD_TYPE', 'FIXED_LENGTH')
  ('RECORD_BYTES', 131072)
  ('FILE_RECORDS', 29)
  ('PRODUCT_ID', 'EUV2001_093_08_35')
  ('SPACECRAFT_CLOCK_START_COUNT', '1/1364978874.96')
  ('SPACECRAFT_CLOCK_STOP_COUNT', 'UNK')
  ('START_TIME',
   datetime.datetime(2001, 4, 3, 8, 35, 20, 430000, tzinfo=datetime.timezone.utc))
  ('STOP_TIME',
   datetime.datetime(2001, 4, 3, 9, 4, 20, 430000, tzinfo=datetime.timezone.utc))
  ('DATA_SET_ID', 'CO-X-UVIS-2-CUBE-V1.2')
  ('TARGET_NAME', 'N/A')
  ('INSTRUMENT_HOST_NAME', 'CASSINI_ORBITER')
  ('INSTRUMENT_NAME', 'ULTRAVIOLET IMAGING SPECTROGRAPH')
  ('INSTRUMENT_ID', 'UVIS')
  ('OBSERVATION_ID', 5150)
  ('MISSION_PHASE_NAME', 'CRUISE')
  ('PRODUCT_CREATION_TIME', datetime.date(2012, 1, 31))
  ('INTEGRATION_DURATION', Quantity(value=60.0, units='SECOND'))
  ('COMPRESSION_TYPE', 'SQRT_9')
  ('HI_VOLTAGE_POWER_SUPPLY_STATE', 'ON')
  ('OCCULTATION_PORT_STATE', 'CLOSED')
  ('SLIT_STATE', 'OCCULTATION')
  ('TEST_PULSE_STATE', 'OFF')
  ('ODC_ID', 93)
  ('DESCRIPTION',
   'This file was produced by the Cassini UVIS team at the Laboratory for '
   'Atmospheric and Space Physics (LASP) at the University of Colorado at '
   'Boulder. For a description of hardware and scientific objectives, refer to '
   'the Cassini UVIS instrument paper The Cassini Ultraviolet Imaging '
   'Spectrograph Investigation, Space Science Reviews, 115, 299-361, by L. W. '
   'Esposito et. al.')
  ('RIGHT_ASCENSION', 'UNK')
  ('DECLINATION', 'UNK')
  ('SUB_SOLAR_LATITUDE', 'UNK')
  ('SUB_SOLAR_LONGITUDE', 'UNK')
  ('SUB_SPACECRAFT_LATITUDE', 'UNK')
  ('SUB_SPACECRAFT_LONGITUDE', 'UNK')
  ('PHASE_ANGLE', 'UNK')
  ('EMISSION_ANGLE', 'UNK')
  ('INCIDENCE_ANGLE', 'UNK')
  ('CENTRAL_BODY_DISTANCE', 'UNK')
  ('SC_PLANET_POSITION_VECTOR', 'UNK')
  ('SC_PLANET_VELOCITY_VECTOR', 'UNK')
  ('SC_SUN_POSITION_VECTOR',
   [Quantity(value=236151430.0, units='KM'),
    Quantity(value=733122610.0, units='KM'),
    Quantity(value=307417460.0, units='KM')])
  ('SC_SUN_VELOCITY_VECTOR',
   [Quantity(value=-5.5180026, units='KM/S'),
    Quantity(value=10.273728, units='KM/S'),
    Quantity(value=4.4153265, units='KM/S')])
  ('SC_TARGET_POSITION_VECTOR', 'UNK')
  ('SC_TARGET_VELOCITY_VECTOR', 'UNK')
  ('PLANET_CENTER_POSITION_VECTOR', 'UNK')
  ('PLANET_CENTER_VELOCITY_VECTOR', 'UNK')
  ('^QUBE', 'EUV2001_093_08_35.DAT')
  ('QUBE',
   {'AXES': 3,
    'AXIS_NAME': ['BAND', 'LINE', 'SAMPLE'],
    'BAND_BIN': 1,
    'BAND_BIN_CENTER': 'N/A',
    'CORE_BASE': 0.0,
    'CORE_HIGH_INSTR_SATURATION': 'N/A',
    'CORE_HIGH_REPR_SATURATION': 'N/A',
    'CORE_ITEMS': [1024, 64, 29],
    'CORE_ITEM_BYTES': 2,
    'CORE_ITEM_TYPE': 'MSB_UNSIGNED_INTEGER',
    'CORE_LOW_INSTR_SATURATION': 'N/A',
    'CORE_LOW_REPR_SATURATION': 'N/A',
    'CORE_MULTIPLIER': 1.0,
    'CORE_NAME': 'RAW_DATA_NUMBER',
    'CORE_NULL': -1,
    'CORE_UNIT': 'COUNTS/BIN',
    'CORE_VALID_MINIMUM': 'N/A',
    'DESCRIPTION': 'This UVIS Qube is a 3 dimensional matrix of 2 byte integers. '
                   'The Qube represents a time series of two dimensional '
                   '(spectral x spatial) matrices. Each two dimensional matrix '
                   'is made up of counts taken by the detector during a fixed '
                   'time interval. The fixed time interval is specified in the '
                   'INTEGRATION_DURATION keyword. If the two dimensional matrix '
                   'has a LINE magnitude of 1 then the Qube is a time series of '
                   'spectra. The region of the detector in which data was taken '
                   'is defined by the ...CORNER keywords. This object is '
                   'organized so that a sequence of 1024 2-byte numbers forms a '
                   'horizontal line (the BAND dimension), a sequence of 64 lines '
                   'forms an BANDxLINE spectral-spatial matrix, and sequence of '
                   '29 of these matrices, forms a (1024x64x29) Qube. The purpose '
                   'of this observation is to measure the response of the EUV '
                   'and FUV channels to various stars for purposes of '
                   'photometric calibration.',
    'LINE_BIN': 1,
    'LR_CORNER_BAND': 1023,
    'LR_CORNER_LINE': 63,
    'SUFFIX_BYTES': 4,
    'SUFFIX_ITEMS': [0, 0, 0],
    'UL_CORNER_BAND': 0,
    'UL_CORNER_LINE': 0})
])
{% endraw %} {% raw %} 
archive_df.loc["EUV2001_093_08_35_28"]

path    /home/maye/uvis_archive/observations/EUV2001_0...
det                                                   EUV
Name: EUV2001_093_08_35_28, dtype: object
{% endraw %} {% raw %} 
import hvplot.xarray
import xarray as xr
{% endraw %} {% raw %} 
ds = xr.open_dataset(fname)
ds

<xarray.Dataset>
Dimensions:   (integrations: 54, spatial_dim_0: 1, spectral_dim_0: 1024)
Dimensions without coordinates: integrations, spatial_dim_0, spectral_dim_0
Data variables:
    window_0  (integrations, spatial_dim_0, spectral_dim_0) int16 0 1 15 ... 0 0
Attributes: (12/16)
    windows:               1.0
    compression:           0
    odc_id:                7
    integration:           32
    channel:               EUV
    hvps_level:            0
    ...                    ...
    stop_time:             
    Version:               1
    start_time_str:        1999-007 17:05:02.000 (1999-Jan-07) SCClock=(12944...
    SCTime:                1294420183
    SCTimeFine:            0
    NetCDFWriter Version:  1.0
{% endraw %} {% raw %} 
np.percentile(ds.window_0, (5, 95))

array([  0., 201.])
{% endraw %} {% raw %} 
ds.window_0.hvplot.image(
    x="spectral_dim_0", y="integrations", cmap="viridis", clim=(0, 201)
)
{% endraw %} {% raw %} 
ds.window_0.mean("integrations").hvplot(x="spectral_dim_0")
{% endraw %} {% raw %} 
ds.window_0.plot()

<matplotlib.collections.QuadMesh at 0x7f01a1bb1790>
{% endraw %} {% raw %} 
p = obsdir / "index_repaired.tab"
{% endraw %} {% raw %} 
df = pd.read_csv(p, quotechar='"', skipinitialspace=True)

/home/maye/miniconda3/envs/py38/lib/python3.8/site-packages/IPython/core/interactiveshell.py:3441: DtypeWarning: Columns (0,9,10,11,12,13) have mixed types.Specify dtype option on import or set low_memory=False.
  exec(code_obj, self.user_global_ns, self.user_ns)
{% endraw %} {% raw %} 
from planetarypy.pds.indexes import find_mixed_type_cols
{% endraw %} {% raw %} 
find_mixed_type_cols(df, fix=False)

9
1999-01-07 10:05:02.093
1999-01-07 10:08:34.0
N/A
EUV1999-01-07 17:05:02.000
Unnamed: 6
USTARE
to evaluate EUV and FUV functions.
7
32
0
0.1
0.2

['9',
 '1999-01-07 10:05:02.093',
 '1999-01-07 10:08:34.0',
 'N/A',
 'EUV1999-01-07 17:05:02.000',
 'Unnamed: 6',
 'USTARE',
 'to evaluate EUV and FUV functions.',
 '7',
 '32',
 '0',
 '0.1',
 '0.2']
{% endraw %} {% raw %} 
find_mixed_type_cols??

Signature: find_mixed_type_cols(df, fix=True)
Source:   
def find_mixed_type_cols(df, fix=True):
    """For a given dataframe, find the columns that are of mixed type.

    Tool to help with the performance warning when trying to save a pandas DataFrame as a HDF.
    When a column changes datatype somewhere, pickling occurs, slowing down the reading process of the HDF file.

    Parameters
    ----------
    df : pandas.DataFrame
        Dataframe to be searched for mixed data-types
    fix : bool
        Switch to control if NaN values in these problem columns should be replaced by the string 'UNKNOWN'
    Returns
    -------
    List of column names that have data type changes within themselves.
    """
    result = []
    for col in df.columns:
        weird = (df[[col]].applymap(type) != df[[col]].iloc[0].apply(type)).any(axis=1)
        if len(df[weird]) > 0:
            print(col)
            result.append(col)
    if fix is True:
        for col in result:
            df[col].fillna("UNKNOWN", inplace=True)
    return result
File:      ~/Dropbox/src/nbplanetary/planetarypy/pds/indexes.py
Type:      function
{% endraw %} {% raw %} 
df.columns

Index(['9', '1999-01-07 10:05:02.093', '1999-01-07 10:08:34.0', 'EUV', 'N/A',
       'EUV1999-01-07 17:05:02.000', 'Unnamed: 6', 'USTARE',
       'to evaluate EUV and FUV functions.', '7', '32', '0', '0.1', '0.2',
       '0.3', '1', '0.4'],
      dtype='object')
{% endraw %} {% raw %} 
index.columns

Index(['FILE_NAME', 'OBSERVATION_TYPE', 'START_TIME', 'STOP_TIME',
       'TARGET_NAME', 'DATA_SET_ID', 'SPACECRAFT_CLOCK_START_COUNT',
       'SPACECRAFT_CLOCK_STOP_COUNT', 'INTEGRATION_DURATION',
       'COMPRESSION_TYPE', 'HI_VOLTAGE_POWER_SUPPLY_STATE',
       'OCCULTATION_PORT_STATE', 'SLIT_STATE', 'TEST_PULSE_STATE', 'ODC_ID',
       'RIGHT_ASCENSION', 'DECLINATION', 'SUB_SOLAR_LATITUDE',
       'SUB_SOLAR_LONGITUDE', 'SUB_SPACECRAFT_LATITUDE',
       'SUB_SPACECRAFT_LONGITUDE', 'PHASE_ANGLE', 'EMISSION_ANGLE',
       'SOLAR_INCIDENCE_ANGLE', 'CENTRAL_BODY_DISTANCE', 'DWELL_TIME',
       'H_LEVEL', 'D_LEVEL', 'filename'],
      dtype='object')
{% endraw %} {% raw %} 
index[index.filename.str.startswith("EUV")].iloc[0]

FILE_NAME                        /COUVIS_0001/DATA/D1999_007/EUV1999_007_17_05.LBL
OBSERVATION_TYPE                                                            USTARE
START_TIME                                              1999-01-07 17:05:01.949000
STOP_TIME                                               1999-01-07 17:08:37.949000
TARGET_NAME                                                                    NaN
DATA_SET_ID                                                  CO-J-UVIS-2-SPEC-V1.2
SPACECRAFT_CLOCK_START_COUNT                                      1/1294420183.000
SPACECRAFT_CLOCK_STOP_COUNT                                                    UNK
INTEGRATION_DURATION                                                           4.0
COMPRESSION_TYPE                                                              NONE
HI_VOLTAGE_POWER_SUPPLY_STATE                                                  OFF
OCCULTATION_PORT_STATE                                                      CLOSED
SLIT_STATE                                                         HIGH_RESOLUTION
TEST_PULSE_STATE                                                                ON
ODC_ID                                                                           7
RIGHT_ASCENSION                                                             -999.0
DECLINATION                                                                 -999.0
SUB_SOLAR_LATITUDE                                                          -999.0
SUB_SOLAR_LONGITUDE                                                         -999.0
SUB_SPACECRAFT_LATITUDE                                                     -999.0
SUB_SPACECRAFT_LONGITUDE                                                    -999.0
PHASE_ANGLE                                                                 -999.0
EMISSION_ANGLE                                                              -999.0
SOLAR_INCIDENCE_ANGLE                                                       -999.0
CENTRAL_BODY_DISTANCE                                                       -999.0
DWELL_TIME                                           1969-12-31 23:59:59.999999001
H_LEVEL                                                                        NaN
D_LEVEL                                                                        NaN
filename                                                         EUV1999_007_17_05
Name: 9, dtype: object
{% endraw %} {% raw %} 
obs.head()
filename slit nx ny nz int odcid x1 x2 y1 y2 name
0 EUV1999_016_19_47_15 2 1024 64 2 60 12 0 1023 0 63 alp vir
1 EUV1999_016_19_49_06 2 1024 1 123 1 13 0 1023 0 63 alp vir
2 EUV1999_016_20_08_10 2 1 64 399 1 11 0 1023 0 63 alp vir
3 EUV1999_016_20_16_14 2 1024 64 2 60 12 0 1023 0 63 alp vir
4 EUV1999_016_20_18_06 2 1024 1 123 1 13 0 1023 0 63 alp vir
{% endraw %} {% raw %} 
cols = ["index start_time stop_time detector target obsid_time unknown type comment "]
{% endraw %}